#!/bin/bash
# sqoop 导入脚本 将MySql 导入到ODS中
export SQOOP_HOME=/usr/bin/sqoop
# 判断传入数据是否为空,如果传入时间为空 则用默认时间,否则 用传入时间
if [ $# -eq 1 ]
then
datastr=$1
else
datastr=`date -d '-1 day' +%Y-%m-%d`
fi
start_time=`date +%Y-%m-%d`
yearstr=`date +%Y`
monthstr=`date +%m`
jdbc_url='jdbc:mysql://hadoop01:3306/nev'
username='root'
password='123456'
# 增量添加主表数据
${SQOOP_HOME} import \
--connect ${jdbc_url} \
--username ${username} --password ${password} \
--query "SELECT 
 id, create_date_time, session_id, sid, create_time, seo_source, seo_keywords, ip,
 area, country, province, city, origin_channel,  user as user_match, manual_time, begin_time,
 end_time, last_customer_msg_time_stamp, last_agent_msg_time_stamp, 
 reply_msg_count, msg_count, browser_name, os_info
 FROM nev.web_chat_ems_${yearstr}_${monthstr} where 1=1 and \$CONDITIONS" \
 --fields-terminated-by '\t' \
 --hcatalog-database macmillanedu_ods \
 --hcatalog-table web_chat_ems \
 -m 3 \
 --split-by id

# 增量添加副表数据
${SQOOP_HOME} import \
--connect ${jdbc_url} \
--username ${username} --password ${password} \
--query "SELECT 
id, referrer, from_url, landing_page_url, url_title, 
platform_description, other_params, history,
'${start_time} 00:00:00' as start_time 
FROM nev.web_chat_text_ems_${yearstr}_${monthstr} 
where 1=1 and \$CONDITIONS" \
 --fields-terminated-by '\t' \
 --hcatalog-database macmillanedu_ods \
 --hcatalog-table web_chat_text_ems \
 -m 3 \
 --split-by id










